# packages for wrangling data and the original models
library(tidyverse)
library(tidymodels)
library(ranger)
library(glmnet)
library(kknn)
library(rcfss)
library(here) # defining consistent filepaths

# packages for model interpretation/explanation
library(DALEX)
library(DALEXtra)

# set random number generator seed value for reproducibility
set.seed(123)
theme_set(theme_minimal())

Import models

# load Rdata file with all the data frames and pre-trained models
load(here("data", "models.Rdata"))

# show pre-trained models
rf_wf
## ══ Workflow [trained] ══════════════════════════════════════════════════════════
## Preprocessor: Recipe
## Model: rand_forest()
## 
## ── Preprocessor ────────────────────────────────────────────────────────────────
## 4 Recipe Steps
## 
## • step_novel()
## • step_impute_median()
## • step_impute_mode()
## • step_naomit()
## 
## ── Model ───────────────────────────────────────────────────────────────────────
## Ranger result
## 
## Call:
##  ranger::ranger(x = maybe_data_frame(x), y = y, num.threads = 1,      verbose = FALSE, seed = sample.int(10^5, 1)) 
## 
## Type:                             Regression 
## Number of trees:                  500 
## Sample size:                      892 
## Number of independent variables:  11 
## Mtry:                             3 
## Target node size:                 5 
## Variable importance mode:         none 
## Splitrule:                        variance 
## OOB prediction error (MSE):       8563494 
## R squared (OOB):                  0.445
glmnet_wf
## ══ Workflow [trained] ══════════════════════════════════════════════════════════
## Preprocessor: Recipe
## Model: linear_reg()
## 
## ── Preprocessor ────────────────────────────────────────────────────────────────
## 7 Recipe Steps
## 
## • step_novel()
## • step_impute_median()
## • step_impute_mode()
## • step_naomit()
## • step_dummy()
## • step_zv()
## • step_normalize()
## 
## ── Model ───────────────────────────────────────────────────────────────────────
## 
## Call:  glmnet::glmnet(x = maybe_matrix(x), y = y, family = "gaussian",      alpha = ~0.05) 
## 
##     Df %Dev Lambda
## 1    0  0.0  39300
## 2    1  0.5  35800
## 3    2  1.1  32700
## 4    2  2.0  29800
## 5    4  2.9  27100
## 6    4  4.2  24700
## 7    6  5.7  22500
## 8    6  7.2  20500
## 9    7  8.7  18700
## 10   7 10.3  17000
## 11   7 11.8  15500
## 12   7 13.3  14100
## 13   7 14.8  12900
## 14   8 16.2  11700
## 15   8 17.6  10700
## 16   8 18.9   9740
## 17   9 20.2   8880
## 18  12 21.6   8090
## 19  13 23.1   7370
## 20  13 24.5   6720
## 21  13 25.8   6120
## 22  14 27.1   5580
## 23  19 28.4   5080
## 24  21 29.7   4630
## 25  23 31.0   4220
## 26  26 32.2   3840
## 27  27 33.4   3500
## 28  29 34.6   3190
## 29  32 35.6   2910
## 30  35 36.6   2650
## 31  37 37.7   2410
## 32  40 38.7   2200
## 33  42 39.6   2000
## 34  45 40.5   1830
## 35  48 41.3   1660
## 36  48 42.1   1520
## 37  49 42.8   1380
## 38  50 43.5   1260
## 39  52 44.1   1150
## 40  53 44.7   1040
## 41  54 45.2    952
## 42  55 45.7    867
## 43  56 46.1    790
## 44  58 46.5    720
## 45  58 46.9    656
## 46  58 47.2    598
## 
## ...
## and 54 more lines.
kknn_wf
## ══ Workflow [trained] ══════════════════════════════════════════════════════════
## Preprocessor: Recipe
## Model: nearest_neighbor()
## 
## ── Preprocessor ────────────────────────────────────────────────────────────────
## 7 Recipe Steps
## 
## • step_novel()
## • step_impute_median()
## • step_impute_mode()
## • step_naomit()
## • step_dummy()
## • step_zv()
## • step_normalize()
## 
## ── Model ───────────────────────────────────────────────────────────────────────
## 
## Call:
## kknn::train.kknn(formula = ..y ~ ., data = data, ks = min_rows(10,     data, 5))
## 
## Type of response variable: continuous
## minimal mean absolute error: 2384
## Minimal mean squared error: 9884682
## Best kernel: optimal
## Best k: 10

Create explainer objects

# use explain_*() to create explainer object
# first step of an DALEX operation
explainer_glmnet <- explain_tidymodels(
  model = glmnet_wf,
  # data should exclude the outcome feature
  data = scorecard_train %>% select(-debt),
  # y should be a vector containing the outcome of interest for the training set
  y = scorecard_train$debt,
  # assign a label to clearly identify model in later plots
  label = "penalized regression"
)
## Preparation of a new explainer is initiated
##   -> model label       :  penalized regression 
##   -> data              :  892  rows  11  cols 
##   -> data              :  tibble converted into a data.frame 
##   -> target variable   :  892  values 
##   -> predict function  :  yhat.workflow  will be used (  default  )
##   -> predicted values  :  No value for predict function target column. (  default  )
##   -> model_info        :  package tidymodels , ver. 0.1.4 , task regression (  default  ) 
##   -> predicted values  :  numerical, min =  4549 , mean =  16564 , max =  23798  
##   -> residual function :  difference between y and yhat (  default  )
##   -> residuals         :  numerical, min =  -9383 , mean =  -1.17e-11 , max =  8285  
##   A new explainer has been created! 

explainer_rf <- explain_tidymodels(
  model = rf_wf,
  data = scorecard_train %>% select(-debt),
  y = scorecard_train$debt,
  label = "random forest"
)
## Preparation of a new explainer is initiated
##   -> model label       :  random forest 
##   -> data              :  892  rows  11  cols 
##   -> data              :  tibble converted into a data.frame 
##   -> target variable   :  892  values 
##   -> predict function  :  yhat.workflow  will be used (  default  )
##   -> predicted values  :  No value for predict function target column. (  default  )
##   -> model_info        :  package tidymodels , ver. 0.1.4 , task regression (  default  ) 
##   -> predicted values  :  numerical, min =  7167 , mean =  16577 , max =  24933  
##   -> residual function :  difference between y and yhat (  default  )
##   -> residuals         :  numerical, min =  -4550 , mean =  -12.9 , max =  5052  
##   A new explainer has been created! 

explainer_kknn <- explain_tidymodels(
  model = kknn_wf,
  data = scorecard_train %>% select(-debt),
  y = scorecard_train$debt,
  label = "k nearest neighbors"
)
## Preparation of a new explainer is initiated
##   -> model label       :  k nearest neighbors 
##   -> data              :  892  rows  11  cols 
##   -> data              :  tibble converted into a data.frame 
##   -> target variable   :  892  values 
##   -> predict function  :  yhat.workflow  will be used (  default  )
##   -> predicted values  :  No value for predict function target column. (  default  )
##   -> model_info        :  package tidymodels , ver. 0.1.4 , task regression (  default  ) 
##   -> predicted values  :  numerical, min =  8406 , mean =  16690 , max =  24768  
##   -> residual function :  difference between y and yhat (  default  )
##   -> residuals         :  numerical, min =  -6956 , mean =  -126 , max =  8328  
##   A new explainer has been created! 

Global interpretation methods

Imputation-based feature importance

# random forest model first
vip_rf <- model_parts(explainer_rf)
plot(vip_rf)


# adjust sampling
## N = 100
model_parts(explainer_rf, N = 100) %>%
  plot()


## all observations
model_parts(explainer_rf, N = NULL) %>%
  plot()


# calculate ratio rather than raw change
model_parts(explainer_rf, type = "ratio") %>%
  plot()

Exercises

  • Calculate feature importance for the penalized regression and \(k\) nearest neighbors using all observations for permutations. How do they compare to the random forest model?
  • Calculate feature importance for the random forest model three times, changing the random seed value before each calculation. How do the results change?
# compare to the glmnet model
vip_glmnet <- model_parts(explainer_glmnet, N = NULL)
plot(vip_glmnet)


# compare to the kknn model
vip_kknn <- model_parts(explainer_kknn, N = NULL)
plot(vip_kknn)


# calculate random forest feature importance thrice
set.seed(123)
model_parts(explainer_rf) %>% plot()


set.seed(234)
model_parts(explainer_rf) %>% plot()


set.seed(345)
model_parts(explainer_rf) %>% plot()

Partial dependence plots

# basic pdp for RF model and netcost variable
#
pdp_netcost <- model_profile(explainer_rf, variables = "netcost")

## just the PDP
plot(pdp_netcost)


## PDP with ICE curves
plot(pdp_netcost, geom = "profiles")


## larger sample size
model_profile(explainer_rf, variables = "netcost", N = 500) %>%
  plot(geom = "profiles")


# group by type
pdp_cost_group <- model_profile(explainer_rf, variables = "netcost", groups = "type", N = NULL)
plot(pdp_cost_group, geom = "profiles")

Exercises

  • Create PDP + ICE curves for netcost using all three models
  • Create a PDP for all numeric variables in the penalized regression model
# create PDP + ICE curves for netcost from all three models
model_profile(explainer_rf, variables = "netcost", N = NULL) %>% plot(geom = "profiles")

model_profile(explainer_glmnet, variables = "netcost", N = NULL) %>% plot(geom = "profiles")

model_profile(explainer_kknn, variables = "netcost", N = NULL) %>% plot(geom = "profiles")


# create PDP for all numeric variables in glmnet model
model_profile(explainer_glmnet) %>%
  plot()

  • Create a PDP for the state variable and the random forest model
# PDP for state
## hard to read
pdp_state_kknn <- model_profile(explainer_kknn, variables = "state", N = NULL)
plot(pdp_state_kknn)


## manually construct and reorder states
## extract aggregated profiles
pdp_state_kknn$agr_profiles %>%
  # convert to tibble
  as_tibble() %>%
  mutate(`_x_` = fct_reorder(.f = `_x_`, .x = `_yhat_`)) %>%
  ggplot(mapping = aes(x = `_yhat_`, y = `_x_`, fill = `_yhat_`)) +
  geom_col() +
  scale_x_continuous(labels = scales::dollar) +
  scale_fill_viridis_c(guide = "none") +
  labs(
    title = "Partial dependence plot for state",
    subtitle = "Created for the k nearest neighbors model",
    x = "Average prediction",
    y = NULL
  )

Local explanation methods

Choose a couple of observations to explain

# filter University of Chicago and Western Illinois University from original dataset
uchi <- filter(.data = scorecard, name == "University of Chicago") %>%
  # remove unitid and name variables since they are not used in the model
  select(-unitid, -name)
wiu <- filter(.data = scorecard, name == "Western Illinois University") %>%
  select(-unitid, -name)

Shapley values

# explain uchicago with rf model
shap_uchi_rf <- predict_parts(
  explainer = explainer_rf,
  new_observation = uchi,
  type = "shap"
)
plot(shap_uchi_rf)


# explain uchicago with kknn model
shap_uchi_kknn <- predict_parts(
  explainer = explainer_kknn,
  new_observation = uchi,
  type = "shap"
)
plot(shap_uchi_kknn)


# increase the number of feature order permutations
predict_parts(
  explainer = explainer_kknn,
  new_observation = uchi,
  type = "shap",
  B = 40
) %>%
  plot()

Pair with ggplot2

# based on example from https://www.tmwr.org/explain.html#local-explanations

shap_uchi_kknn %>%
  # convert to pure tibble-formatted data frame
  as_tibble() %>%
  # calculate average contribution per variable across permutations
  group_by(variable) %>%
  mutate(mean_val = mean(contribution)) %>%
  ungroup() %>%
  # reorder variable levels in order of absolute value of mean contribution
  mutate(variable = fct_reorder(variable, abs(mean_val))) %>%
  # define basic ggplot object for horizontal boxplot
  ggplot(mapping = aes(x = contribution, y = variable, fill = mean_val > 0)) +
  # add a bar plot
  geom_col(
    data = ~ distinct(., variable, mean_val),
    mapping = aes(x = mean_val, y = variable),
    alpha = 0.5
  ) +
  # overlay with boxplot to show distribution
  geom_boxplot(width = 0.5) +
  # outcome variable is measured in dollars - contributions are the same units
  scale_x_continuous(labels = scales::dollar) +
  # use viridis color palette
  scale_fill_viridis_d(guide = "none") +
  labs(y = NULL)

Exercises

  • Explain each model’s prediction for Western Illinois University. How do they differ?
# calculate shapley values
shap_wiu_rf <- predict_parts(
  explainer = explainer_rf,
  new_observation = wiu,
  type = "shap"
)

shap_wiu_kknn <- predict_parts(
  explainer = explainer_kknn,
  new_observation = wiu,
  type = "shap"
)

shap_wiu_glmnet <- predict_parts(
  explainer = explainer_glmnet,
  new_observation = wiu,
  type = "shap"
)

# generate plots for each
plot(shap_wiu_rf)

plot(shap_wiu_kknn)

plot(shap_wiu_glmnet)


# view side by side
library(patchwork)
plot(shap_wiu_rf) +
  plot(shap_wiu_kknn) +
  plot(shap_wiu_glmnet)


# or combine together and reuse ggplot code from above
bind_rows(
  shap_wiu_rf,
  shap_wiu_kknn,
  shap_wiu_glmnet
) %>%
  # convert to pure tibble-formatted data frame
  as_tibble() %>%
  # calculate average contribution per variable across permutations
  group_by(label, variable) %>%
  mutate(mean_val = mean(contribution)) %>%
  ungroup() %>%
  # reorder variable levels in order of absolute value of mean contribution
  # mutate(variable = fct_reorder(variable, abs(mean_val))) %>%
  mutate(variable = tidytext::reorder_within(x = variable, by = abs(mean_val), within = label)) %>%
  # define basic ggplot object for horizontal boxplot
  ggplot(mapping = aes(x = contribution, y = variable, fill = mean_val > 0)) +
  # add a bar plot
  geom_col(
    data = ~ distinct(., label, variable, mean_val),
    mapping = aes(x = mean_val, y = variable),
    alpha = 0.5
  ) +
  # overlay with boxplot to show distribution
  geom_boxplot(width = 0.5) +
  # facet for each model
  facet_wrap(vars(label), scales = "free_y") +
  tidytext::scale_y_reordered() +
  # outcome variable is measured in dollars - contributions are the same units
  scale_x_continuous(labels = scales::dollar) +
  # use viridis color palette
  scale_fill_viridis_d(guide = "none") +
  labs(y = NULL)

LIME

# load LIME package - note conflict with DALEX::explain()
library(lime)

# prepare the recipe
prepped_rec_rf <- extract_recipe(rf_wf)

# write a function to bake the observation
bake_rf <- function(x) {
  bake(
    prepped_rec_rf,
    new_data = x
  )
}

# create explainer object
lime_explainer_rf <- lime(
  x = scorecard_train,
  model = extract_fit_parsnip(rf_wf),
  preprocess = bake_rf
)

# top 5 features
explanation_rf <- explain(
  x = uchi,
  explainer = lime_explainer_rf,
  n_features = 5
)

plot_features(explanation_rf)


# top 10 features, increased permutations
explanation_rf <- explain(
  x = uchi,
  explainer = lime_explainer_rf,
  n_features = 10,
  n_permutations = 2000
)

plot_features(explanation_rf)

Exercises

  • Calculate a LIME explanation for Western Illinois and the \(k\) nearest neighbors model. What are the top 10 features? How well does the local model explain the prediction?
  • Reproduce the explanation but use a lasso model to select the most important features. How does the explanation change?
# prepare the recipe
prepped_rec_kknn <- extract_recipe(kknn_wf)

# write a function to bake the observation
bake_kknn <- function(x) {
  bake(
    prepped_rec_kknn,
    new_data = x
  )
}

# create explainer object
lime_explainer_kknn <- lime(
  x = scorecard_train,
  model = extract_fit_parsnip(kknn_wf),
  preprocess = bake_kknn
)

# top 10 features
explanation_kknn <- explain(
  x = uchi,
  explainer = lime_explainer_kknn,
  n_features = 10
)

plot_features(explanation_kknn)


# use lasso to select the most important features
explanation_lasso_kknn <- explain(
  x = uchi,
  explainer = lime_explainer_kknn,
  n_features = 10,
  feature_select = "lasso_path"
)

plot_features(explanation_lasso_kknn)

A note on the penalized regression model

Due to how the model was trained, bake_glmnet() requires an additional composition argument. Otherwise everything else is the same.

# prepare the recipe
prepped_rec_glmnet <- extract_recipe(glmnet_wf)

# write a function to convert the legislative description to an appropriate matrix object
bake_glmnet <- function(x) {
  bake(
    prepped_rec_glmnet,
    new_data = x,
    composition = "dgCMatrix"
  )
}

# create explainer object
lime_explainer_glmnet <- lime(
  x = scorecard_train,
  model = extract_fit_parsnip(glmnet_wf),
  preprocess = bake_glmnet
)

# top 5 features
explanation_glmnet <- explain(
  x = uchi,
  explainer = lime_explainer_glmnet,
  n_features = 5)

plot_features(explanation_glmnet)

Session Info

sessioninfo::session_info()
## ─ Session info ───────────────────────────────────────────────────────────────
##  setting  value
##  version  R version 4.1.2 (2021-11-01)
##  os       macOS Monterey 12.1
##  system   aarch64, darwin20
##  ui       X11
##  language (EN)
##  collate  en_US.UTF-8
##  ctype    en_US.UTF-8
##  tz       America/Chicago
##  date     2022-02-15
##  pandoc   2.14.2 @ /usr/local/bin/ (via rmarkdown)
## 
## ─ Packages ───────────────────────────────────────────────────────────────────
##  package      * version    date (UTC) lib source
##  assertthat     0.2.1      2019-03-21 [1] CRAN (R 4.1.0)
##  backports      1.4.1      2021-12-13 [1] CRAN (R 4.1.1)
##  broom        * 0.7.12     2022-01-28 [1] CRAN (R 4.1.1)
##  bslib          0.3.1      2021-10-06 [1] CRAN (R 4.1.1)
##  cellranger     1.1.0      2016-07-27 [1] CRAN (R 4.1.0)
##  class          7.3-20     2022-01-13 [1] CRAN (R 4.1.1)
##  cli            3.1.1      2022-01-20 [1] CRAN (R 4.1.1)
##  codetools      0.2-18     2020-11-04 [1] CRAN (R 4.1.2)
##  colorspace     2.0-2      2021-06-24 [1] CRAN (R 4.1.1)
##  crayon         1.4.2      2021-10-29 [1] CRAN (R 4.1.1)
##  DALEX        * 2.3.0      2021-07-28 [1] CRAN (R 4.1.0)
##  DALEXtra     * 2.1.1      2021-05-09 [1] CRAN (R 4.1.0)
##  DBI            1.1.2      2021-12-20 [1] CRAN (R 4.1.1)
##  dbplyr         2.1.1      2021-04-06 [1] CRAN (R 4.1.0)
##  dials        * 0.0.10     2021-09-10 [1] CRAN (R 4.1.1)
##  DiceDesign     1.9        2021-02-13 [1] CRAN (R 4.1.0)
##  digest         0.6.29     2021-12-01 [1] CRAN (R 4.1.1)
##  dplyr        * 1.0.7      2021-06-18 [1] CRAN (R 4.1.0)
##  ellipsis       0.3.2      2021-04-29 [1] CRAN (R 4.1.0)
##  evaluate       0.14       2019-05-28 [1] CRAN (R 4.1.0)
##  fansi          1.0.2      2022-01-14 [1] CRAN (R 4.1.1)
##  farver         2.1.0      2021-02-28 [1] CRAN (R 4.1.0)
##  fastmap        1.1.0      2021-01-25 [1] CRAN (R 4.1.0)
##  forcats      * 0.5.1      2021-01-27 [1] CRAN (R 4.1.1)
##  foreach        1.5.1      2020-10-15 [1] CRAN (R 4.1.0)
##  fs             1.5.2      2021-12-08 [1] CRAN (R 4.1.1)
##  furrr          0.2.3      2021-06-25 [1] CRAN (R 4.1.0)
##  future         1.23.0     2021-10-31 [1] CRAN (R 4.1.1)
##  future.apply   1.8.1      2021-08-10 [1] CRAN (R 4.1.1)
##  generics       0.1.2      2022-01-31 [1] CRAN (R 4.1.1)
##  ggplot2      * 3.3.5      2021-06-25 [1] CRAN (R 4.1.1)
##  glmnet       * 4.1-3      2021-11-02 [1] CRAN (R 4.1.1)
##  globals        0.14.0     2020-11-22 [1] CRAN (R 4.1.0)
##  glue           1.6.1      2022-01-22 [1] CRAN (R 4.1.1)
##  gower          0.2.2      2020-06-23 [1] CRAN (R 4.1.0)
##  GPfit          1.0-8      2019-02-08 [1] CRAN (R 4.1.0)
##  gtable         0.3.0      2019-03-25 [1] CRAN (R 4.1.1)
##  hardhat        0.2.0      2022-01-24 [1] CRAN (R 4.1.1)
##  haven          2.4.3      2021-08-04 [1] CRAN (R 4.1.1)
##  here         * 1.0.1      2020-12-13 [1] CRAN (R 4.1.0)
##  highr          0.9        2021-04-16 [1] CRAN (R 4.1.0)
##  hms            1.1.1      2021-09-26 [1] CRAN (R 4.1.1)
##  htmltools      0.5.2      2021-08-25 [1] CRAN (R 4.1.1)
##  httr           1.4.2      2020-07-20 [1] CRAN (R 4.1.0)
##  iBreakDown     2.0.1      2021-05-07 [1] CRAN (R 4.1.1)
##  igraph         1.2.11     2022-01-04 [1] CRAN (R 4.1.1)
##  infer        * 1.0.0      2021-08-13 [1] CRAN (R 4.1.1)
##  ingredients    2.2.0      2021-04-10 [1] CRAN (R 4.1.1)
##  ipred          0.9-12     2021-09-15 [1] CRAN (R 4.1.1)
##  iterators      1.0.13     2020-10-15 [1] CRAN (R 4.1.0)
##  janeaustenr    0.1.5      2017-06-10 [1] CRAN (R 4.1.0)
##  jquerylib      0.1.4      2021-04-26 [1] CRAN (R 4.1.0)
##  jsonlite       1.7.3      2022-01-17 [1] CRAN (R 4.1.1)
##  kknn         * 1.3.1      2016-03-26 [1] CRAN (R 4.1.0)
##  knitr          1.37       2021-12-16 [1] CRAN (R 4.1.1)
##  labeling       0.4.2      2020-10-20 [1] CRAN (R 4.1.0)
##  lattice        0.20-45    2021-09-22 [1] CRAN (R 4.1.2)
##  lava           1.6.10     2021-09-02 [1] CRAN (R 4.1.1)
##  lhs            1.1.3      2021-09-08 [1] CRAN (R 4.1.1)
##  lifecycle      1.0.1      2021-09-24 [1] CRAN (R 4.1.1)
##  lime         * 0.5.2      2021-02-24 [1] CRAN (R 4.1.1)
##  listenv        0.8.0      2019-12-05 [1] CRAN (R 4.1.0)
##  lubridate      1.8.0      2021-10-07 [1] CRAN (R 4.1.1)
##  magrittr       2.0.2      2022-01-26 [1] CRAN (R 4.1.1)
##  MASS           7.3-55     2022-01-13 [1] CRAN (R 4.1.1)
##  Matrix       * 1.4-0      2021-12-08 [1] CRAN (R 4.1.1)
##  modeldata    * 0.1.1      2021-07-14 [1] CRAN (R 4.1.0)
##  modelr         0.1.8      2020-05-19 [1] CRAN (R 4.1.0)
##  munsell        0.5.0      2018-06-12 [1] CRAN (R 4.1.0)
##  nnet           7.3-17     2022-01-13 [1] CRAN (R 4.1.1)
##  parallelly     1.30.0     2021-12-17 [1] CRAN (R 4.1.1)
##  parsnip      * 0.1.7      2021-07-21 [1] CRAN (R 4.1.0)
##  patchwork    * 1.1.1      2020-12-17 [1] CRAN (R 4.1.1)
##  pillar         1.6.5      2022-01-25 [1] CRAN (R 4.1.2)
##  pkgconfig      2.0.3      2019-09-22 [1] CRAN (R 4.1.0)
##  plyr           1.8.6      2020-03-03 [1] CRAN (R 4.1.0)
##  png            0.1-7      2013-12-03 [1] CRAN (R 4.1.0)
##  pROC           1.18.0     2021-09-03 [1] CRAN (R 4.1.1)
##  prodlim        2019.11.13 2019-11-17 [1] CRAN (R 4.1.0)
##  purrr        * 0.3.4      2020-04-17 [1] CRAN (R 4.1.0)
##  R6             2.5.1      2021-08-19 [1] CRAN (R 4.1.1)
##  ranger       * 0.13.1     2021-07-14 [1] CRAN (R 4.1.0)
##  rcfss        * 0.2.3      2022-02-10 [1] local
##  Rcpp           1.0.8      2022-01-13 [1] CRAN (R 4.1.1)
##  readr        * 2.1.1      2021-11-30 [1] CRAN (R 4.1.1)
##  readxl         1.3.1      2019-03-13 [1] CRAN (R 4.1.0)
##  recipes      * 0.1.17     2021-09-27 [1] CRAN (R 4.1.1)
##  reprex         2.0.1      2021-08-05 [1] CRAN (R 4.1.1)
##  reticulate     1.24-9000  2022-02-09 [1] Github (rstudio/reticulate@273c98a)
##  rlang          1.0.1      2022-02-03 [1] CRAN (R 4.1.1)
##  rmarkdown      2.11       2021-09-14 [1] CRAN (R 4.1.1)
##  rpart          4.1.16     2022-01-24 [1] CRAN (R 4.1.1)
##  rprojroot      2.0.2      2020-11-15 [1] CRAN (R 4.1.0)
##  rsample      * 0.1.1      2021-11-08 [1] CRAN (R 4.1.1)
##  rstudioapi     0.13       2020-11-12 [1] CRAN (R 4.1.0)
##  rvest          1.0.2      2021-10-16 [1] CRAN (R 4.1.1)
##  sass           0.4.0      2021-05-12 [1] CRAN (R 4.1.0)
##  scales       * 1.1.1      2020-05-11 [1] CRAN (R 4.1.0)
##  sessioninfo    1.2.2      2021-12-06 [1] CRAN (R 4.1.1)
##  shape          1.4.6      2021-05-19 [1] CRAN (R 4.1.0)
##  SnowballC      0.7.0      2020-04-01 [1] CRAN (R 4.1.0)
##  stringi        1.7.6      2021-11-29 [1] CRAN (R 4.1.1)
##  stringr      * 1.4.0      2019-02-10 [1] CRAN (R 4.1.1)
##  survival       3.2-13     2021-08-24 [1] CRAN (R 4.1.2)
##  tibble       * 3.1.6      2021-11-07 [1] CRAN (R 4.1.1)
##  tidymodels   * 0.1.4      2021-10-01 [1] CRAN (R 4.1.1)
##  tidyr        * 1.1.4      2021-09-27 [1] CRAN (R 4.1.1)
##  tidyselect     1.1.1      2021-04-30 [1] CRAN (R 4.1.0)
##  tidytext       0.3.2      2021-09-30 [1] CRAN (R 4.1.1)
##  tidyverse    * 1.3.1      2021-04-15 [1] CRAN (R 4.1.0)
##  timeDate       3043.102   2018-02-21 [1] CRAN (R 4.1.0)
##  tokenizers     0.2.1      2018-03-29 [1] CRAN (R 4.1.0)
##  tune         * 0.1.6      2021-07-21 [1] CRAN (R 4.1.0)
##  tzdb           0.2.0      2021-10-27 [1] CRAN (R 4.1.1)
##  utf8           1.2.2      2021-07-24 [1] CRAN (R 4.1.0)
##  vctrs          0.3.8      2021-04-29 [1] CRAN (R 4.1.0)
##  viridisLite    0.4.0      2021-04-13 [1] CRAN (R 4.1.0)
##  withr          2.4.3      2021-11-30 [1] CRAN (R 4.1.1)
##  workflows    * 0.2.4      2021-10-12 [1] CRAN (R 4.1.1)
##  workflowsets * 0.1.0      2021-07-22 [1] CRAN (R 4.1.1)
##  xfun           0.29       2021-12-14 [1] CRAN (R 4.1.1)
##  xml2           1.3.3      2021-11-30 [1] CRAN (R 4.1.1)
##  yaml           2.2.2      2022-01-25 [1] CRAN (R 4.1.1)
##  yardstick    * 0.0.9      2021-11-22 [1] CRAN (R 4.1.1)
## 
##  [1] /Library/Frameworks/R.framework/Versions/4.1-arm64/Resources/library
## 
## ──────────────────────────────────────────────────────────────────────────────